import requests
from bs4 import BeautifulSoup as bs
from pyquery import PyQuery as pq
import json
import re
from sql_db_bd import MySqlHelper
USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36'
headers = {
    'User-Agent': USER_AGENT
}


def read_bd_list(url, item_dict_all=None):
    if item_dict_all is None:
        item_dict_all = list()
    response = requests.get(url, headers=headers)
    if response.status_code == 200:
        movie_1 = json.loads(response.text)
        num = 0
        class_1 = movie_1['res']['payload']
        classs = class_1.get('title')
        movie_name = movie_1['res']
        mo = movie_name.get('subjects')
        if classs:
            if mo:
                for i in mo:
                    num += 1
                    fenlei = classs
                    name = i['title']
                    goods = i['rating']
                    img = i['cover']
                    url = i['url']
                    paiming = num
                    item_dict_all.append([fenlei,name, goods, img, url, paiming])

            else:
                return
        else:
            return
    else:
        print("你抓的出问题了！")
    return item_dict_all


if __name__ == "__main__":
    for i in range(1, 65):
        url_1 = "https://movie.douban.com/ithil_j/activity/movie_annual2018/widget/" + str(i)
        item_dict_all = read_bd_list(url_1)
        mysql = MySqlHelper()
        sql = "INSERT INTO movies(fenlei, title, rating, img, url, num) VALUES(%s,%s,%s,%s,%s,%s)"
        print(item_dict_all)
        if item_dict_all:
            mysql.exec_many(sql, item_dict_all)